import re
from httplib2 import Http
from util import get_db

http = Http()

db = get_db()


def crawl(site_url):
    post_re = re.compile(site_url + '(\d\d\d\d)/(\d\d)/(\d\d)/[^/]/')

    def get(url):
        headers, content = http.request(url)
        if post_re.match(url):
            db.pages.insert(dict(
                url=url,
                content=content
            ))
        for url in find
